{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# Varying regularization in Multi-layer Perceptron\n",
    "\n",
    "\n",
    "A comparison of different values for regularization parameter 'alpha' on\n",
    "synthetic datasets. The plot shows that different alphas yield different\n",
    "decision functions.\n",
    "\n",
    "Alpha is a parameter for regularization term, aka penalty term, that combats\n",
    "overfitting by constraining the size of the weights. Increasing alpha may fix\n",
    "high variance (a sign of overfitting) by encouraging smaller weights, resulting\n",
    "in a decision boundary plot that appears with lesser curvatures.\n",
    "Similarly, decreasing alpha may fix high bias (a sign of underfitting) by\n",
    "encouraging larger weights, potentially resulting in a more complicated\n",
    "decision boundary.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatically created module for IPython interactive environment\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Kieron\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:563: ConvergenceWarning: Stochastic Optimizer: Maximum iterations reached and the optimization hasn't converged yet.\n",
      "  % (), ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "print(__doc__)\n",
    "\n",
    "\n",
    "# Author: Issam H. Laradji\n",
    "# License: BSD 3 clause\n",
    "\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib.colors import ListedColormap\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.datasets import make_moons, make_circles, make_classification\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "h = .02  # step size in the mesh\n",
    "\n",
    "alphas = np.logspace(-5, 3, 5)\n",
    "names = []\n",
    "for i in alphas:\n",
    "    names.append('alpha ' + str(i))\n",
    "\n",
    "classifiers = []\n",
    "for i in alphas:\n",
    "    classifiers.append(MLPClassifier(alpha=i, random_state=1))\n",
    "\n",
    "X, y = make_classification(n_features=2, n_redundant=0, n_informative=2,\n",
    "                           random_state=0, n_clusters_per_class=1)\n",
    "rng = np.random.RandomState(2)\n",
    "X += 2 * rng.uniform(size=X.shape)\n",
    "linearly_separable = (X, y)\n",
    "\n",
    "datasets = [make_moons(noise=0.3, random_state=0),\n",
    "            make_circles(noise=0.2, factor=0.5, random_state=1),\n",
    "            linearly_separable]\n",
    "\n",
    "figure = plt.figure(figsize=(17, 9))\n",
    "i = 1\n",
    "# iterate over datasets\n",
    "for X, y in datasets:\n",
    "    # preprocess dataset, split into training and test part\n",
    "    X = StandardScaler().fit_transform(X)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.4)\n",
    "\n",
    "    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5\n",
    "    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5\n",
    "    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n",
    "                         np.arange(y_min, y_max, h))\n",
    "\n",
    "    # just plot the dataset first\n",
    "    cm = plt.cm.RdBu\n",
    "    cm_bright = ListedColormap(['#FF0000', '#0000FF'])\n",
    "    ax = plt.subplot(len(datasets), len(classifiers) + 1, i)\n",
    "    # Plot the training points\n",
    "    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright)\n",
    "    # and testing points\n",
    "    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6)\n",
    "    ax.set_xlim(xx.min(), xx.max())\n",
    "    ax.set_ylim(yy.min(), yy.max())\n",
    "    ax.set_xticks(())\n",
    "    ax.set_yticks(())\n",
    "    i += 1\n",
    "\n",
    "    # iterate over classifiers\n",
    "    for name, clf in zip(names, classifiers):\n",
    "        ax = plt.subplot(len(datasets), len(classifiers) + 1, i)\n",
    "        clf.fit(X_train, y_train)\n",
    "        score = clf.score(X_test, y_test)\n",
    "\n",
    "        # Plot the decision boundary. For that, we will assign a color to each\n",
    "        # point in the mesh [x_min, x_max]x[y_min, y_max].\n",
    "        if hasattr(clf, \"decision_function\"):\n",
    "            Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])\n",
    "        else:\n",
    "            Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]\n",
    "\n",
    "        # Put the result into a color plot\n",
    "        Z = Z.reshape(xx.shape)\n",
    "        ax.contourf(xx, yy, Z, cmap=cm, alpha=.8)\n",
    "\n",
    "        # Plot also the training points\n",
    "        ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright)\n",
    "        # and testing points\n",
    "        ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright,\n",
    "                   alpha=0.6)\n",
    "\n",
    "        ax.set_xlim(xx.min(), xx.max())\n",
    "        ax.set_ylim(yy.min(), yy.max())\n",
    "        ax.set_xticks(())\n",
    "        ax.set_yticks(())\n",
    "        ax.set_title(name)\n",
    "        ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'),\n",
    "                size=15, horizontalalignment='right')\n",
    "        i += 1\n",
    "\n",
    "figure.subplots_adjust(left=.02, right=.98)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
